#!/usr/bin/env python
#-*-encoding:utf-8-*-
'''
Created on 2014年10月29日

@author: chenyongbing
'''

import os,tarfile,sys
import re,time,datetime,commands
import logging,logging.config
logging.basicConfig()
from localConfig import LocalConfig
myLocalConfig = LocalConfig()
current_dir = os.path.split(os.path.realpath(__file__))[0]
sys.path.append(os.path.join(current_dir,'../'))
from base.PyMySQL import ConnectMySQL
from base.localConfig import baseLocalConfig
from base.time_method import myTimeMethod
from base.retail_datas import myRetailDatas

retail_host = baseLocalConfig.mysqlConfig['retail']['host']
retail_user = baseLocalConfig.mysqlConfig['retail']['user']
retail_passwd = baseLocalConfig.mysqlConfig['retail']['passwd']
retail_db = baseLocalConfig.mysqlConfig['retail']['db']

myRetail = ConnectMySQL(host=retail_host, user=retail_user, password=retail_passwd, db=retail_db)

zhimind_host = baseLocalConfig.mysqlConfig['zhimind']['host']
zhimind_user = baseLocalConfig.mysqlConfig['zhimind']['user']
zhimind_passwd = baseLocalConfig.mysqlConfig['zhimind']['passwd']
zhimind_db = baseLocalConfig.mysqlConfig['zhimind']['db']

myZhimind = ConnectMySQL(host=zhimind_host, user=zhimind_user, password=zhimind_passwd, db=zhimind_db)


import logging




class FilterData(object):
    '''
    classdocs
    '''


    def __init__(self,day=None):
        '''
        Constructor
        '''
        
        #self.GroupsDevices = myAnalyzeConfig.getGroupsDevices()
        self.GroupsDevices = myRetailDatas.get_devices_by_group()
        if day==None:
            self.day = (datetime.date.today()-datetime.timedelta(1)).strftime("%Y-%m-%d")
        else:
            self.day = day
        

        
    def run_command(self,cmd):
        rets = commands.getstatusoutput(cmd)
        if rets[0]==0:
             return True
        errMsg =rets[1]
        logging.error('Run cmd %s Faild.'%cmd)
        logging.error(errMsg)
        return False
        
    def filter_by_device(self,rawfile):
        yesterday = (datetime.date.today() - datetime.timedelta(1)).strftime('%Y-%m-%d')
        rawpath = os.path.dirname(rawfile)
        rfile = re.sub(rawpath,'',rawfile)
        try:
            os.popen('mkdir %s/tmp'%rawpath)
        except:pass
        
        new_rawfile = rawpath+'/tmp/'+re.sub(rawpath,'',rawfile)
        day = re.search('([\d]{4}-[\d]{2}-[\d]{2})',rawfile).group(1)
        while 1:
            if day == yesterday:
                scpCmd = 'scp /data/java/data/rawdata/netty/%s/* %s'%(day,rawpath+'/tmp/')
            elif os.path.exists(rawfile+'.tar.gz'):
                
                tarcmd = 'tar -xzvf %s -C %s'%(rawfile+'.tar.gz',rawpath+'/tmp/')
                logging.debug(tarcmd)
                if self.run_command(tarcmd):break
            elif os.path.exists('/Udisk/rawdata/%s.tar.gz'%rfile):
                tarcmd = 'tar -xzvf %s -C %s'%('/Udisk/rawdata/%s.tar.gz'%rfile,rawpath+'/tmp/')
                logging.debug(tarcmd)
                if self.run_command(tarcmd):break
            else:
                logging.error('RawFile %s.tar.gz Not Exists.'%rawfile)
                logging.error('RawFile /Udisk/rawdata/%s.tar.gz Not Exists.'%rfile)
                sys.exit()
        os.chdir(rawpath+'/tmp')
        
        logging.debug('rm -rf %s/[!rawdata]*'%(rawpath+'/tmp'))
        self.run_command('rm -rf %s/[!rawdata]*'%(rawpath+'/tmp'))
        result = True
        for new_rawfile in os.listdir(rawpath+'/tmp'):
            cmd = "awk -F '.' '{if ($2!=null) print $0 >>$2}' %s"%new_rawfile
            if not self.run_command(cmd):
                logging.error('Filter rawfile %s Faild.'%rawfile)
                result = False
        return result
    def merger_device_rawfile_by_group(self,rawfile='',group=None,filter_device=True):
        if filter_device:
            self.filter_by_device(rawfile)
        
        rawpath = os.path.dirname(rawfile)
        os.chdir(rawpath+'/tmp')
        group_rawpath = rawpath+'/rawdata_group'
        if not os.path.exists(group_rawpath):
            os.mkdir(group_rawpath)
        group_rawpath_day = group_rawpath+'/'+self.day
        if not os.path.exists(group_rawpath_day):
            os.mkdir(group_rawpath_day)
            
        if group in self.GroupsDevices.keys():
            myGroupsDevices = {group:self.GroupsDevices[group]}
        else:
            myGroupsDevices = self.GroupsDevices
        for group , devices in myGroupsDevices.iteritems():
            group = str(group)
            delDevs = []
            for dev in devices:
                if not os.path.exists(dev):
                    logging.error('Device %s Not Exists Rawdata.'%dev)
                    delDevs.append(dev)
            #devices.remove(dev)
            devices = list(set(devices) - set(delDevs))
            if len(devices)==0:continue
            
            cmd = 'tar -czvf %s %s'%(group_rawpath_day+'/'+str(group)+'.txt.tar.gz',' '.join(devices))
            for i in range(5):
                if self.run_command(cmd):break
            if not os.path.exists(group_rawpath_day+'/'+str(group)+'.txt.tar.gz'):
                logging.error('Tar Raw File Faild Groups(%s) Device(%s).'%(group,','.join(devices)))
            for device in devices:
                delCmd = 'rm -rf %s'%device
                self.run_command(delCmd)
        if os.listdir(rawpath+'/tmp/')>0:
            tarOtherCmd = 'tar -czvf %s *'%(group_rawpath_day+'/others.tar.gz')
            self.run_command(tarOtherCmd)
        if filter_device:
            commands.getstatusoutput('rm -rf %s'%(rawpath+'/tmp'))
  
        
        
if __name__=='__main__':
    
    from logging.handlers import RotatingFileHandler

    logging.basicConfig(level= logging.INFO)
    
    Rthandler = RotatingFileHandler('/data/log/filter_rrawdata.log', maxBytes=100*1024*1024,backupCount=0)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    Rthandler.setFormatter(formatter)
    logging.getLogger('').addHandler(Rthandler)
    
    console = logging.StreamHandler()
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)
    import argparse,re,sys
    parser = argparse.ArgumentParser(description='args')
    parser.add_argument('--day',metavar=u"day",default=None)
    parser.add_argument('--group',metavar=u"group",default=None)
    parser.add_argument('--filterType',metavar=u"device",default='device')
    parser.add_argument('--startTime',metavar=u"day",default=None)
    parser.add_argument('--endTime',metavar=u"day",default=None)
    parser.add_argument('--nofilter',action='store_true',help='no filter')
    
    args = parser.parse_args()
    day = args.day
    group = args.group
    filterType = args.filterType
    startTime = args.startTime
    endTime = args.endTime
    nofilter = args.nofilter
    #day = '2014-10-15'
    if day==None:
        day = (datetime.datetime.today()-datetime.timedelta(1)).strftime('%Y-%m-%d')
        days = [day]
    else:
        days = [day]
    if startTime!=None and endTime!=None:
        days = myTimeMethod.get_slist_between_st_et(startTime, endTime)
    for day in days:
        myFilterData = FilterData(day)
    
        
        rawpath = myLocalConfig.rawdata_path
        rawfile = rawpath+'/rawdata.%s.txt'%day
        if filterType=='device':
            filter_device = True
        else:
            filter_device = False
        if nofilter:
            filter_device = False
        myFilterData.merger_device_rawfile_by_group(rawfile,group= group,filter_device=filter_device)
    
       

    